# -*- coding: utf-8 -*-
import scrapy
import re
from fang.items import ESFHouseItem, NewHouseItem
from scrapy_redis.spiders import RedisSpider


class FtxSpider(RedisSpider):
    name = 'ftx'
    allowed_domains = ['fang.com']
    # start_urls = ['https://www.fang.com/SoufunFamily.htm']
    redis_key = "fang:start_urls"
    def parse(self, response):
        trs = response.xpath("//div[@class='outCont']/table//tr")
        province = None
        for tr in trs:
            tds = tr.xpath(".//td[not(@class)]")
            province_td = tds[0]
            province_text = province_td.xpath(".//text()").get()
            province_text = re.sub(r"\s", "", province_text)
            if province_text:
                province = province_text
            # 不需要海外房价信息
            if province == "其它":
                continue
            city_td = tds[1]
            city_links = city_td.xpath(".//a")
            for city_link in city_links:
                city = city_link.xpath(".//text()").get()
                city_url = city_link.xpath(".//@href").get()
                url_model = city_url.split(".")
                # 二手房链接
                url_model.insert(1, "esf")
                city_esf_url = ".".join(url_model)
                # 北京二手房链接特殊处理
                if "bj." in city_esf_url:
                    city_esf_url = "https://esf.fang.com/"
                # 新房链接
                url_model[1] = "newhouse"
                city_newhouse_url = ".".join(url_model) + "house/s/"
                # 北京新房链接特殊处理
                if "bj." in city_esf_url:
                    city_newhouse_url = "https://newhouse.fang.com/house/s/"
                # 二手房
                yield scrapy.Request(url=city_esf_url, callback=self.parse_esf, meta={"info":(province, city)})
                # 新房
                yield scrapy.Request(url=city_newhouse_url, callback=self.parse_newhouse, meta={"info":(province, city)})

    def parse_esf(self, response):
        province, city = response.meta.get("info")
        dls = response.xpath("//div[contains(@class, 'shop_list')]/dl")
        for dl in dls:
            item = ESFHouseItem(province=province, city=city)
            name = dl.xpath(".//p[@class='add_shop']/a/@title").get()
            address = dl.xpath(".//p[@class='add_shop']/span/text()").get()
            item["name"] = name
            item["address"] = address
            infos = dl.xpath(".//p[@class='tel_shop']/text()").getall()
            infos = list(map(lambda x:re.sub(r"\s", "", x), infos))
            for info in infos:
                if "厅" in info:
                    item["rooms"] = info
                elif "层" in info:
                    item["floor"] = info
                elif "向" in info:
                    item["toward"] = info
                elif "㎡" in info:
                    item["area"] = info
                elif "年" in info:
                    item["year"] = info.replace("年建：", "")
                else:
                    pass
            item["price"] = price = dl.xpath(".//dd[@class='price_right']/span[1]/b/text()").get()
            if item["price"]:
                item["price"] += "万"
            item["unit"] = unit = dl.xpath(".//dd[@class='price_right']/span[2]/text()").get()
            detail_url = dl.xpath(".//dt/a/@href").get()
            item["origin_url"] = response.urljoin(detail_url)
            yield item
        # 下一页
        pager = response.xpath("//div[@id='list_D10_15']/p")
        if len(pager) == 5:
            next_page_url = pager[-3].xpath(".//a/@href").get()
            next_page_url = response.urljoin(next_page_url)
            yield scrapy.Request(url=next_page_url, callback=self.parse_esf, meta={"info": (province, city)})


    def parse_newhouse(self, response):
        lis = response.xpath("//div[@id='newhouse_loupai_list']/ul/li")
        province, city = response.meta.get("info")
        for li in lis:
            item = NewHouseItem(province=province, city=city)
            name = li.xpath(".//div[@class='nlcd_name']/a/text()").get()
            origin_url = li.xpath(".//div[@class='nlcd_name']/a/@href").get()
            if name:
                name = name.strip()
            item["name"] = name
            item["origin_url"] = origin_url
            address = li.xpath(".//div[@class='address']/a/@title").get()
            item["address"] = address
            unit = li.xpath(".//div[@class='nhouse_price']/span/text()").get()
            if unit and "待定" not in unit:
                unit += "元/㎡"
            else:
                unit = "待定"
            item["unit"] = unit
            sale = li.xpath(".//div[contains(@class, 'fangyuan')]/span/text()").get()
            item["sale"] = sale
            yield item
        # 翻页
        if "b9" not in response.url:
            final_page_url = response.xpath("//div[@class='page']/ul/li/a[last()]/@href").get()
            print(final_page_url, "=================")
            if final_page_url and re.search(r"b9(\d+?)/$", final_page_url):
                final_page_re = re.search(r"b9(\d+?)/$", final_page_url)
                final_page = int(final_page_re.group(1))
                # response.urljoin是以域名为基础连接的，不包含路径
                url_model = response.urljoin("/house/s/b9{}/")
                for page in range(2, final_page + 1):
                    page_url = url_model.format(page)
                    yield scrapy.Request(url=page_url, callback=self.parse_newhouse, meta={"info": (province, city)})





